# -*- coding: utf-8 -*-
import scrapy

import re
import json
import logging
from copy import copy, deepcopy
from FDC_spider.items import FdcEstateGuidItem, FdcBuildingItem, FdcRoomItem

logger = logging.getLogger(__name__)


class TongrenSpider(scrapy.Spider):
    name = 'tongren'
    allowed_domains = ['trfcj.com']
    start_urls = ['http://www.trfcj.com/Client/Nanjiang/Second/Second_HouseManger.aspx?RelationCID=&PageSize=5']
    # GET 项目列表url
    project_li_url_temp = 'http://www.trfcj.com/Client/Nanjiang/Scripts/Paging/PagingHandler.ashx?MLandAgentName=&ProjectName=&ProjectAddress=&PrePressionCertNo=&&act=Project&columnID=&curPage={}&pageSize=20'
    # GET 项目详情url
    project_url_temp = 'http://www.trfcj.com/Client/Nanjiang/Second/Detail/ProjectInfo/ProjectDetail.aspx?id={}&RelationCID=&PageSize=5'
    # GET 开发商详情url
    company_url_temp = 'http://www.trfcj.com/Client/Nanjiang/Second/Detail/EnterpriseInfo/EnterpriseAndProjectDetails.aspx?id={}&RelationCID=&PageSize=5'

    def start_requests(self):
        """
        重构项目列表页首页请求
        :return:
        """
        headers: dict = {
            'Referer': self.start_urls[0],
        }
        page_num: int = 1
        yield scrapy.Request(
            self.project_li_url_temp.format(page_num, ),
            headers=headers,
            meta=dict(page_num=page_num, ),
        )

    def parse(self, response):
        """
        项目列表页
        :param response:
        :return:
        """
        page_num = copy(response.meta['page_num'])
        try:
            resp_dict: dict = json.loads(response.body.decode())
            # 提取总页数
            total_page_num: int = int(resp_dict.get('TotalPage', None))
            # 提取当前页项目列表信息
            records_li: list = resp_dict.get('Records', None)
            assert records_li, f'第{page_num}页  项目列表获取为空'
        except AssertionError as e:
            logger.error(e)
        except Exception as e:
            logger.error(f'第{page_num}页  项目列表获取出错，msg:{e if e else "error"}')
        else:
            # 遍历，获取项目信息
            for project_dict in records_li:
                item_eg = FdcEstateGuidItem()
                item_eg['projectName'] = project_dict.get('ProjectName', None)  # 项目名称
                item_eg['projectId'] = project_dict.get('ProjectId', None)  # 项目id
                item_eg['developerName'] = project_dict.get('MLandAgentName', None)  # 开发商
                company_id = project_dict.get('MLandAgentCode', None)  # 开发商id（用于构造url，不入库）
                item_eg['preSalePermit'] = project_dict.get('PrePressionCertNo', None)  # 预售许可证
                item_eg['projectAddress'] = project_dict.get('ProjectAddress', None)  # 项目地址
                item_eg['totalRoomNo'] = project_dict.get('TotalNumber', None)  # 总套数
                item_eg['totalArea'] = project_dict.get('PlanArea', None)  # 总面积
                item_eg['projectFileNum'] = project_dict.get('ProjectFileNum', None)  # 立项文号
                # 获取项目详情
                if item_eg['projectId']:
                    headers: dict = {
                        'Referer': self.start_urls[0],
                    }
                    yield scrapy.Request(
                        self.project_url_temp.format(item_eg['projectId']),
                        headers=headers,
                        callback=self.parse_project_detail,
                        meta=dict(item_eg=deepcopy(item_eg), company_id=deepcopy(company_id), ),
                        priority=20,
                    )
                else:
                    logger.error('第{}页-{} 项目id提取为空，无法访问项目详情'.format(page_num, item_eg['projectName']))

            #  翻页
            if page_num < total_page_num:
                headers: dict = {
                    'Referer': self.start_urls[0],
                }
                page_num += 1
                yield scrapy.Request(
                    self.project_li_url_temp.format(page_num, ),
                    headers=headers,
                    meta=dict(page_num=page_num, ),
                    priority=10,
                )

    def parse_project_detail(self, response):
        """
        获取项目详情
        :param response:
        :return:
        """
        item_eg = copy(response.meta['item_eg'])
        company_id = copy(response.meta['company_id'])
        # 提取项目详情信息
        item_eg['projectUrl'] = response.request.url  # 项目url
        item_eg['projectNature'] = response.xpath(
            '//td[contains(text(),"主体性质")]/following-sibling::td[1]/text()').extract_first()  # 项目性质
        item_eg['planningCommencementDate'] = response.xpath(
            '//td[contains(text(),"计划开工日期")]/following-sibling::td[1]/text()').extract_first()  # 计划开工日期
        item_eg['planningCompletionDate'] = response.xpath(
            '//td[contains(text(),"计划竣工日期")]/following-sibling::td[1]/text()').extract_first()  # 计划竣工日期
        item_eg['totalResidenceArea'] = response.xpath(
            '//td[contains(text(),"住宅建筑面积")]/following-sibling::td[1]/text()').extract_first()  # 住宅总面积
        item_eg['totalNonResidenceArea'] = response.xpath(
            '//td[contains(text(),"非住宅建筑面积")]/following-sibling::td[1]/text()').extract_first()  # 非住宅总面积
        item_eg['districtName'] = response.xpath(
            '//td[contains(text(),"项目坐落")]/following-sibling::td[1]/text()').extract_first()  # 项目区域/行政区划
        item_eg['publicName'] = response.xpath(
            '//td[contains(text(),"销售名称")]/following-sibling::td[1]/text()').extract_first()  # 推广名称
        item_eg['saleAddress'] = response.xpath(
            '//td[contains(text(),"售楼地址")]/following-sibling::td[1]/text()').extract_first()  # 销售地址
        item_eg['salesTel'] = response.xpath(
            '//td[contains(text(),"售楼电话")]/following-sibling::td[1]/text()').extract_first()  # 销售电话
        item_eg['openingDate'] = response.xpath(
            '//td[contains(text(),"开盘日期")]/following-sibling::td[1]/text()').extract_first()  # 开盘日期
        item_eg['handoverDate'] = response.xpath(
            '//td[contains(text(),"入住日期")]/following-sibling::td[1]/text()').extract_first()  # 上房/交房/交付日期
        item_eg['parkingSpacesNum'] = response.xpath(
            '//td[contains(text(),"车位个数")]/following-sibling::td[1]/text()').extract_first()  # 停车位
        item_eg['totalBuildingNum'] = response.xpath(
            '//td[contains(text(),"总栋数")]/following-sibling::td[1]/text()').extract_first()  # 总楼栋数
        item_eg['projectBrief'] = response.xpath(
            '//td[contains(text(),"项目简介")]/following-sibling::td[1]/text()').extract_first()  # 项目简介
        item_eg['landGrade'] = response.xpath(
            '//td[contains(text(),"土地等级")]/following-sibling::td[1]/text()').extract_first()  # 土地等级
        item_eg['commencementDate'] = response.xpath(
            '//td[contains(text(),"实际开工时间")]/following-sibling::td[1]/text()').extract_first()  # 开工日期
        item_eg['coverageArea'] = response.xpath(
            '//td[contains(text(),"占地面积")]/following-sibling::td[1]/text()').extract_first()  # 占地面积

        #  获取楼栋列表分组
        tr_li = response.xpath("//table[@id='table1']//tbody/tr")
        if tr_li:
            # 遍历提取楼栋信息
            for tr_html_obj in tr_li:
                item_bd = FdcBuildingItem()
                item_bd['projectName'] = item_eg['projectName']  # 项目名称
                item_bd['projectId'] = item_eg['projectId']  # 项目id
                item_bd['districtName'] = item_eg['districtName']  # 项目区域/行政区划
                item_bd['buildingUrl'] = tr_html_obj.xpath('./td[1]/a/@href').extract_first()  # 楼栋url
                item_bd['buildingId'] = self.get_building_id(item_bd['buildingUrl'])  # 楼栋id
                item_bd['blockName'] = tr_html_obj.xpath('./td[2]/text()').extract_first()  # 楼栋名字
                item_bd['approvalRoomNo'] = tr_html_obj.xpath('./td[3]/text()').extract_first()  # 批准套数
                item_bd['approvalArea'] = tr_html_obj.xpath('./td[4]/text()').extract_first()  # 批准面积
                item_bd['residenceAvgPrice'] = tr_html_obj.xpath('./td[5]/text()').extract_first()  # 住宅均价
                item_bd['nonResidenceAvgPrice'] = tr_html_obj.xpath('./td[6]/text()').extract_first()  # 非住宅均价
                # 获取房号列表
                if item_bd['buildingUrl']:
                    yield scrapy.Request(
                        item_bd['buildingUrl'],
                        callback=self.parse_room_li,
                        meta=dict(item_bd=deepcopy(item_bd), ),
                        priority=30,
                    )
                else:
                    logger.error('{}-{} 楼栋url提取为空，url：{}'.format(item_bd['projectName'], item_bd['blockName'],
                                                                 response.request.url))
        else:
            logger.warning('{} 楼栋列表提取为空，url：{}'.format(item_eg['projectName'], response.request.url))

        # 获取开发商详情
        if company_id:
            headers: dict = {
                'Referer': self.start_urls[0],
            }
            yield scrapy.Request(
                self.company_url_temp.format(company_id),
                headers=headers,
                callback=self.parse_company_detail,
                meta=dict(item_eg=deepcopy(item_eg), ),
                priority=40,
                dont_filter=True,
            )
        else:
            logger.warning('{} 开发商id提取为空，无法获取开发商详情信息'.format(item_eg['projectName']))
            item_eg['qualificationGrade'] = None  # 资质等级
            item_eg['qualificationCertificateNo'] = None  # 资质证书编号
            yield item_eg

    def parse_company_detail(self, response):
        """
        获取开发商详情
        :param response:
        :return:
        """
        item_eg = copy(response.meta['item_eg'])
        item_eg['qualificationGrade'] = response.xpath(
            "//td[contains(text(),'资质等级')]/following-sibling::td[1]/text()").extract_first()  # 资质等级
        item_eg['qualificationCertificateNo'] = response.xpath(
            "//td[contains(text(),'资质证编号')]/following-sibling::td[1]/text()").extract_first()  # 资质证书编号
        yield item_eg

    def parse_room_li(self, response):
        """
        获取房号列表
        :param response:
        :return:
        """
        item_bd = copy(response.meta['item_bd'])
        # 提取房号列表信息
        room_li = self.get_room_li(response.body.decode(), item_bd)
        # 校验房号数量
        room_div = response.xpath("//body/div[1]/div[@id!='']")
        if len(room_li) != len(room_div):
            logger.warning(
                '{}-{} 房号数量校验不一致，{}/{}，url：{}'.format(item_bd['projectName'], item_bd['blockName'], len(room_li),
                                                      len(room_div), response.request.url))
        unit_set = set()  # 单元集合
        floor_set = set()  # 楼层集合
        # 遍历获取房号信息
        for room_dict in room_li:
            item_rm = FdcRoomItem()
            item_rm['projectName'] = item_bd['projectName']
            item_rm['projectId'] = item_bd['projectId']
            item_rm['blockName'] = item_bd['blockName']
            item_rm['buildingId'] = item_bd['buildingId']
            item_rm['roomUse'] = room_dict.get('LayoutTypeName', None)  # 房屋用途
            item_rm['roomStructure'] = room_dict.get('DoorTypeName', None)  # 房屋套型/户型
            item_rm['roomDecoration'] = room_dict.get('FinishGradeName', None)  # 装修情况
            item_rm['roomLocation'] = room_dict.get('DoorPlate', None)  # 房屋坐落
            item_rm['roomNo'] = room_dict.get('HouseName', None)  # 房号
            item_rm['roomId'] = room_dict.get('HouseId', None)  # 房屋id
            item_rm['buildingStructure'] = room_dict.get('StruTypeName', None)  # 建筑结构
            item_rm['roomNature'] = room_dict.get('HousePropertyName', None)  # 房屋性质
            item_rm['forecastBuildArea'] = room_dict.get('PreBuildArea', None)  # 预测建筑面积
            item_rm['forecastInnerArea'] = room_dict.get('PreInnerArea', None)  # 预测套内面积
            item_rm['forecastSharedArea'] = room_dict.get('PreApportionArea', None)  # 预测分摊面积
            item_rm['forecastVerandaArea'] = room_dict.get('PreBalconyArea', None)  # 预测阳台面积
            item_rm['nominalFloor'] = room_dict.get('OnNominalLayer', None)  # 名义层
            item_rm['physicsFloor'] = room_dict.get('PhysicLayer', None)  # 物理层
            item_rm['pubPrice'] = room_dict.get('PrePrice', None)  # 公示价格
            item_rm['verandaCount'] = room_dict.get('BalconyCount', None)  # 阳台个数
            item_rm['isElevator'] = room_dict.get('IsLift', None)  # 是否有电梯
            item_rm['isDuplex'] = room_dict.get('IsLeap', None)  # 是否跃层
            item_rm['buildDate'] = room_dict.get('BuildDate', None)  # 建成日期
            item_rm['floorTotalNo'] = room_dict.get('LayerCount', None)  # 总层数
            item_rm['floorHeight'] = room_dict.get('LayerHeight', None)  # 层高
            item_rm['structure'] = room_dict.get('DoorStruName', None)  # 户型结构
            item_rm['structure'] = room_dict.get('DoorStruName', None)  # 户型结构
            item_rm['unitNo'] = room_dict.get('UnitNum', None)  # 单元号
            unit_set.add(item_rm['unitNo']) if item_rm['unitNo'] else False
            floor_set.add(item_rm['floorTotalNo']) if not floor_set else False
            # 解析获取房号销售状态
            sale_status_str = response.xpath(
                '//div[contains(@id,"{}")]/@style'.format(item_rm['roomId'])).extract_first()
            item_rm['saleStatus'] = self.get_sale_status(sale_status_str)  # 销售状态
            yield item_rm

        # 楼栋单元列表
        item_bd['unitLi'] = list(unit_set) if unit_set else [1]  # 单元列表
        item_bd['floorTotalNo'] = list(floor_set)[0] if floor_set else None  # 总层数
        yield item_bd

    @staticmethod
    def get_building_id(_str: str):
        """
        提取楼栋id
        :param _str: 楼栋url
        :return:
        """
        try:
            ret = _str.split('id=')[1]
            assert ret
        except:
            return
        else:
            return ret

    @staticmethod
    def get_room_li(_str: str, item) -> list:
        """
        提取房号列表
        :param _str: 响应字符串
        :param item: 楼栋item
        :return:
        """
        regex = re.compile(r'houselist = eval[(](.*?)[)];', re.S)
        try:
            room_info_str = regex.findall(_str)[0]
            room_li = json.loads(room_info_str, strict=False)
            assert isinstance(room_li, list), '{}-{} 房号信息反序列化后，类型不是list'.format(item['projectName'], item['blockName'])
        except AssertionError as e:
            logger.error(e)
        except Exception as e:
            logger.error('{}-{} 提取房号列表出错，msg：{}'.format(item['projectName'], item['blockName'], e))
            return list()
        else:
            return room_li

    @staticmethod
    def get_sale_status(status_str: str):
        """
        获取房号销售状态
        :param status_str:
        :return:
        """
        status_dict = {
            '#8DCB57': '未售',
            '#6D71BD': '签约',
            '#7E2CAC': '备案',
            '#0F83B4': '预告',
            '#B41416': '初登',
            '#B6790F': '转移',
            '#80827F': '限制',
        }
        try:
            assert status_str
        except:
            return
        else:
            for key, value in status_dict.items():
                if key in status_str:
                    return value
